package gpl.pierrick.brihaye.aramorph.lucene;
import gpl.pierrick.brihaye.aramorph.AraMorph;

import java.io.IOException;
import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;

import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;

/**
 * A filter that normalizes arabic words to some simpler format.
 * It removes all accents (tashkil) and kashida (tatweel).
 * Also it changes some letters to others.
 * <uol>
 * <li>All types of hamza are changed to alef</li>
 * <li>Zay is changed to Thal</li>
 * <li>Teh marboota is changed to heh marboots</li>
 * <li>Dad is changed to Zah</li>
 * <li>Alef Maksoura (Aleb layyena) is changed to yeh</li>
 * </uol>
 * These rules overcome the most spelling mistakes people do.
 * @author Ahmed Saad
 */
public class ArabicNormalizer extends TokenFilter {

	private static Map<Character,Character> normalizeMap = getNormalizer();
	
	private PositionIncrementAttribute posIncrAtt;
	private TermAttribute termAtt;

	public ArabicNormalizer(TokenStream input) {
		super(input);
		termAtt = (TermAttribute) addAttribute(TermAttribute.class);
		posIncrAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class);
	}

	private static Map<Character, Character> getNormalizer() {
		normalizeMap = new HashMap<Character,Character>();
		//all kinds of alef and hamza
		normalizeMap.put(AraMorph.ARABIC_LETTER_ALEF_WITH_MADDA_ABOVE,AraMorph.ARABIC_LETTER_ALEF);//alef with madda above is turned into alef
		normalizeMap.put(AraMorph.ARABIC_LETTER_HAMZA,AraMorph.ARABIC_LETTER_HAMZA);//hamza
		normalizeMap.put(AraMorph.ARABIC_LETTER_ALEF_WITH_HAMZA_ABOVE,AraMorph.ARABIC_LETTER_ALEF);//alef with hamza above is turned into alef
		normalizeMap.put('\u0624','\u0624');//waw with hamza above
		normalizeMap.put('\u0625','\u0627');//alef with hamza below is turned into alef
		normalizeMap.put('\u0626','\u0626');//yeh with hamza above
		normalizeMap.put('\u0627','\u0627');//alef
		normalizeMap.put('\u0628','\u0628');//beh
		normalizeMap.put('\u0629','\u0629');//teh marboota
		normalizeMap.put('\u062A','\u062A');//teh
		normalizeMap.put('\u062B','\u062B');//theh
		normalizeMap.put('\u062C','\u062C');//jeem
		normalizeMap.put('\u062D','\u062D');//hah
		normalizeMap.put('\u062E','\u062E');//khah
		normalizeMap.put('\u062F','\u062F');//dal
		normalizeMap.put('\u0630','\u0630');//thal
		normalizeMap.put('\u0631','\u0631');//reh
		normalizeMap.put('\u0632','\u0630');//zay is turned into thal
		normalizeMap.put('\u0633','\u0633');//seen
		normalizeMap.put('\u0634','\u0634');//sheen
		normalizeMap.put('\u0635','\u0635');//sad
		normalizeMap.put('\u0636','\u0636');//dad
		normalizeMap.put('\u0637','\u0637');//tah
		normalizeMap.put('\u0638','\u0638');//zah
		normalizeMap.put('\u0639','\u0639');//ayn
		normalizeMap.put('\u063A','\u063A');//ghayn
		normalizeMap.put('\u0641','\u0641');//feh
		normalizeMap.put('\u0642','\u0642');//qaf
		normalizeMap.put('\u0643','\u0643');//kaf
		normalizeMap.put('\u0644','\u0644');//lam
		normalizeMap.put('\u0645','\u0645');//meem
		normalizeMap.put('\u0646','\u0646');//noon
		normalizeMap.put('\u0647','\u0647');//heh
		normalizeMap.put('\u0648','\u0648');//waw
		normalizeMap.put('\u0649','\u064A');//yeh is turned into alef maksoura
		normalizeMap.put('\u064A','\u064A');//alef maksoura (batta)
		return normalizeMap;
	}

	@Override
	public boolean incrementToken() throws IOException {
		//stores position increment in case some tokens are dropped
		int positionIncrement = 0;
		while (input.incrementToken()) {
			//extract the term text and normalize it
			String normalizedText = clean(termAtt.term());
			//add the position increment of the dropped token 
			positionIncrement += posIncrAtt.getPositionIncrement();
			if (normalizedText != null) {
				termAtt.setTermBuffer(normalizedText);
				posIncrAtt.setPositionIncrement(positionIncrement);
				return true;
			}
		}
		return false;
	}
	
	protected String clean(String text) {
		return ArabicNormalizer.normalize(text);
	}
	
	/**
	 * This function normalizes an Arabic text for most common errors done by people.
	 * This helps indexing it in a normalized form so that they can be searched easily.
	 * @param text the text to be normalized.
	 */
	public static String normalize(String text) {
		char[] normalizedText = new char[text.length()];
		int i=0;
		char[] arGloss = text.toCharArray();
		
		for (int x = 0;x < arGloss.length;x++) {
			char c = arGloss[x];
			//do not normalize numbers
			if (Character.isDigit(c))
				return text;
			Character result = normalizeMap.get(new Character(c));
			if (result != null)
				normalizedText[i++] = result.charValue();
		}
		return i!=0?new String(normalizedText,0,i):null;
	}

	public static void main(String[] args) {
		try {
			String arabicText = "أنا إسمي أحمد";
			ArabicTokenizer at = new ArabicTokenizer(new StringReader(arabicText));
			ArabicNormalizer an = new ArabicNormalizer(at);
			while (an.incrementToken()) {
//				System.out.print(an.getAttribute(PositionIncrementAttribute.class).getPositionIncrement());
//				System.out.print(':');
//				System.out.println(an.getAttribute(TermAttribute.class).term());
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

}
